/*
 * @(#)HtmlParserLinkExtractor.java	1.0 05/01/07
 *
 * Copyright 2007 Fabio Gasparetti. All rights reserved.
 */

package org.tabularium.text.html;

import java.io.*;
import java.util.*;

import org.tabularium.net.UrlNormalizer;
import org.tabularium.text.analysis.*;
import org.htmlparser.*;
import org.htmlparser.lexer.Lexer;
import org.htmlparser.util.Translate;
import org.htmlparser.lexer.*;
import org.htmlparser.tags.*;
import org.htmlparser.visitors.*;
import org.htmlparser.beans.*;

/**
 * Implementation of the abstract class LinkExtractor based on the HtmlParser library.
 * 
 * @see LinkExtractor
 *
 * @author  Fabio Gasparetti
 * @version 1.0, 05/01/07
 */

public class HtmlParserLinkExtractor extends LinkExtractor {

	protected HashSet htmlNotFlowBreakSet = new HashSet();	

	public HtmlParserLinkExtractor() {
		for (int i = 0; i < htmlNotFlowBreak.length; i++) {
			this.htmlNotFlowBreakSet.add(htmlNotFlowBreak[i]);
		}		
	}
	
	/**

	 * The current implementation does not consider possible layout separations
	 * in the text, i.e., table's cells, paragraphs, etc., during the retrieval
	 * of the surrounding text.
	 * <p>
	 * Note: the Web page must be escaped-character free, the returned urls are
	 * not normalized.
	 *
	 * @see HtmlEscapeSequence#removeEscapedCh()
	 */
	public List extractLinkWindows(String url, String htmltext, int winSize,
			Analyzer lexicalAnalyzer) throws Exception {
		Lexer lexer;
		org.htmlparser.Parser parser;
		List list = null;

		lexer = new org.htmlparser.lexer.Lexer(htmltext);
		parser = new org.htmlparser.Parser(lexer);
		LinkWindowsVisitor visitor = new LinkWindowsVisitor(url, winSize,
				lexicalAnalyzer);
		visitor.htmlNotFlowBreakSet = htmlNotFlowBreakSet;
		parser.visitAllNodesWith(visitor);
		list = visitor.getList();
		return list;
	}

	/**
	 * Returns an ordered list of urls (repetition allowed) corresponding to
	 * the links in the given Web page.
	 * <p>
	 * Note: the Web page must be escaped-character free, the returned urls are
	 * not normalized.
	 *
	 * @see HtmlEscapeSequence#removeEscapedCh()
	 */
	public List extractLinks(String url, String htmltext) throws Exception {
		Lexer lexer;
		org.htmlparser.Parser parser;
		List list = null;

		lexer = new org.htmlparser.lexer.Lexer(htmltext);
		parser = new org.htmlparser.Parser(lexer);
		LinksVisitor visitor = new LinksVisitor(url);
		parser.visitAllNodesWith(visitor);
		list = visitor.getList();
		return list;
	}

	// for debugging
	public static void main(String[] args) {
		try {
			InputStream is = new FileInputStream("prova2.html");
			BufferedReader reader = new BufferedReader(
					new InputStreamReader(is));

			String line;
			StringBuffer doc = new StringBuffer();
			while ((line = reader.readLine()) != null) {
				doc.append(line + "\n");
			}
			is.close();
			HtmlParserLinkExtractor le = new HtmlParserLinkExtractor(); 
	        List l = le.extractLinks("http://www/", doc.toString());
	        Iterator iter = l.iterator();
	        while (iter.hasNext()) {
	          String item = (String) iter.next();
	          System.out.println(item);

	        }
	        System.out.println("\n\n----\n\n");
	        org.tabularium.text.analysis.StandardAnalyzer a = new org.tabularium.text.analysis.StandardAnalyzer();
	        a.enablePorterStemmer(false);
			List l1 = le.extractLinkWindows("http://www/", doc.toString(), 5, a);
	        Iterator iter1 = l1.iterator();
			while (iter1.hasNext()) {
		          String[] s  = (String[]) iter1.next();
		          System.out.print("\nbefore:'"+s[0]+"' ");
		          System.out.print("\nafter:'"+s[1]+"' ");
		          System.out.print("\nanchor:'"+s[2]+"' ");
		          System.out.print("\ntitle:'"+s[3]+"' ");
		          System.out.print("\nurl:'"+s[4]+"' ");
		          System.out.println("\n---");
		        }

		} catch (Exception ex) {
			System.out.println(ex.toString());
			ex.printStackTrace();
		}
	}
}

/* HtmlParser visitor for link-window extraction. See extractLinkWindows(). */
class LinkWindowsVisitor extends StringBean {
	protected ArrayList list = new ArrayList();

	protected StringBuffer textBeforeLink = null;

	protected StringBuffer textAfterLink = null;

	protected StringBuffer linkText = new StringBuffer();

	protected StringBuffer titleText = new StringBuffer();

	protected StringBuffer buffer = new StringBuffer();

	protected String linkUrl = null;

	protected String passedLinkUrl = null;

	protected int winSize;

	protected Analyzer lexicalAnalyzer;

	protected String url = null;

	protected String baseHref = null;
	
	protected HashSet htmlNotFlowBreakSet = null;		

	public LinkWindowsVisitor(String url, int size, Analyzer analyzer) {
		super();
		baseHref = url;
		this.url = url;
		winSize = size;
		lexicalAnalyzer = analyzer;
	}

	public List getList() {
		return list;
	}

	public void visitEndTag(Tag tag) {
		//System.out.println("CLOSING-TAG: "+tag.getTagName());
		if (breaksFlow(tag)) {
			//System.out.println("BREAK FLOW");
			saveLink();
		}
	}
	
	public void visitTag(Tag tag) {
		//System.out.println("TAG: "+tag.getTagName());
		if (breaksFlow(tag))
			buffer.append(" ");

		if (tag instanceof BaseHrefTag) {
			BaseHrefTag baseHrefTag = (BaseHrefTag) tag;
			baseHref = baseHrefTag.getBaseUrl();
			if (baseHref == null) {
				baseHref = url;
			}
		} else if (tag instanceof TitleTag) {
			// ignore title text
		} else if (tag instanceof ImageTag) {
			//System.out.println(tag.toString());
			ImageTag imageTag = (ImageTag)tag;
			// adds the alt-attribute text of the images between a link tag
			String altAttr = imageTag.getAttribute("ALT");
			if (altAttr == null) {
				return;
			}
			if (insideLinkTag(imageTag)) {
				translateAppend(linkText, altAttr);
				return;
			}
		} else if (tag instanceof LinkTag) {
			newLink((LinkTag)tag);
			//System.out.println(tag.toString());

			//linkUrl = linkTag.getLink();
		} else if (breaksFlow(tag)) {
			//System.out.println("BREAK on TAG: "+tag.getTagName()+","+tag.getClass().getName()+" : "+passedLinkUrl);
			saveLink();
		} 
		
		/*
		if (tag.getTagName().equals("TABLE") || tag.getTagName().equals("TR")
				|| tag.getTagName().equals("TD")
				|| tag.getTagName().equals("TH")
				|| tag.getTagName().equals("UL")
				|| tag.getTagName().equals("OL")
				|| tag.getTagName().equals("LI")) {
			buffer.append(" . ");
		}
		*/
	}
	
	void newLink(LinkTag linkTag) {
		if (passedLinkUrl == null) { // first link
			textBeforeLink = buffer;
			passedLinkUrl = linkTag.getLink();
			titleText = new StringBuffer();
			String titleAttr = linkTag.getAttribute("TITLE");
			if (titleAttr != null) {
				translateAppend(titleText, titleAttr);
			} 
			//linkUrl = passedLinkUrl;
		} else {
			textAfterLink = buffer;
			String u = UrlNormalizer.unrelativizeUrl(baseHref, passedLinkUrl);
			if (u != null) {
				try {
					String[] linkData = LinkExtractor.buildLinkWindow(
							textBeforeLink.toString(), textAfterLink.toString(),
							linkText.toString(), titleText.toString(), u, winSize, lexicalAnalyzer);
					list.add(linkData);
				} catch(IOException ex) { /* lexical analyzer exception */ }
			}
			linkText = new StringBuffer();				
			titleText = new StringBuffer();
			String titleAttr = linkTag.getAttribute("TITLE");
			if (titleAttr != null) {
				translateAppend(titleText, titleAttr);
			} 
			textBeforeLink = textAfterLink;
			textAfterLink = null;
			passedLinkUrl = linkTag.getLink();
		}
		buffer = new StringBuffer();
	}
	
	void saveLink() {
		// same code as we found a new link
		//System.out.println(tag.toString());
		if (passedLinkUrl == null) { // first link
			textBeforeLink = new StringBuffer();
		} else {
			textAfterLink = buffer;
			String u = UrlNormalizer.unrelativizeUrl(baseHref, passedLinkUrl);
			if (u != null) {
				try {
					String[] linkData = LinkExtractor.buildLinkWindow(
							textBeforeLink.toString(), textAfterLink.toString(),
							linkText.toString(), titleText.toString(), u, winSize, lexicalAnalyzer);
					list.add(linkData);
				} catch(IOException ex) { /* lexical analyzer exception */ }
			}
			linkText = new StringBuffer();				
			titleText = new StringBuffer();
			textBeforeLink = new StringBuffer();
			textAfterLink = null;
			passedLinkUrl = null;
		}
		buffer = new StringBuffer();
		//linkUrl = linkTag.getLink();
	}
	
	public boolean breaksFlow(Tag t) {
		boolean ret = true;
		if (htmlNotFlowBreakSet.contains(t.getTagName()))
				ret = false;
		return ret; // || t.breaksFlow();
	}

	public void visitStringNode(Text stringNode) {
		StringBuffer sb = buffer;
		Node parent = stringNode.getParent();
		if ((parent != null)
				&& (parent instanceof org.htmlparser.tags.ScriptTag)) {
			return;
		}
		if ((parent != null)
				&& (parent instanceof org.htmlparser.tags.StyleTag)) {
			return;
		}
		//System.out.println("stringNode: "+stringNode.toString()+" insideLinkTag:"+insideLinkTag(stringNode));
		if (insideLinkTag(stringNode)) 
			sb = linkText;
		
		if (!mIsScript && !mIsStyle) {
			String text = stringNode.getText();
			if (!mIsPre) {
				translateAppend(sb, text);
			} else
				sb.append(text);
		}
	}
	
	public void translateAppend(StringBuffer sb, String text) {
		text = Translate.decode(text);
		if (getReplaceNonBreakingSpaces())
			text = text.replace('\u00a0', ' ');
		if (getCollapse())
			collapse(sb, text);
		else
			sb.append(text);	
	}	
	
	public boolean insideLinkTag(Node t) {
		Node parent = t.getParent();
		if (parent != null) { 
			if (parent instanceof LinkTag) 
				return true;
			return insideLinkTag(parent);
		}
		return false;
	}
	
	public void finishedParsing() {
		//System.out.println("finished parsing");
		if (passedLinkUrl != null) {
			saveLink();
			/*
			textAfterLink = buffer;
			String u = UrlNormalizer.unrelativizeUrl(baseHref, passedLinkUrl);
			if (u != null) {
				try {
					String[] linkData = LinkExtractor.buildLinkWindow(
							textBeforeLink.toString(), textAfterLink.toString(),
							linkText.toString(), titleText.toString(), u, winSize, lexicalAnalyzer);
					list.add(linkData);
				} catch(IOException ex) {}
			} */
		}
	}

}

/* HtmlParser visitor for link-extraction. See extractLinks(). */
class LinksVisitor extends NodeVisitor {
	protected ArrayList list = new ArrayList();

	protected String url = null;

	protected String baseHref = null;

	public LinksVisitor(String url) {
		super(true);
		baseHref = url;
		this.url = url;
	}

	public List getList() {
		return list;
	}

	public void visitTag(Tag tag) {
		if (tag instanceof LinkTag) {
			String linkUrl = UrlNormalizer.unrelativizeUrl(this.baseHref, ((LinkTag)tag).getLink());
			if (linkUrl != null) {
				list.add(linkUrl);
			}
		}
	}

}
